package org.luosl.webmagicx.pipeline.component


import org.eclipse.jetty.util.ConcurrentHashSet
import us.codecraft.webmagic.ResultItems

/**
  * 基于hashset的单机去重机制
  * Created by luosl on 2017/12/14.
  */
class HashSetDistinct(loadCache: ConcurrentHashSet[Any] => Unit,distinctValue:ResultItems => Any) extends Distinct(distinctValue){

  private val set:ConcurrentHashSet[Any] = new ConcurrentHashSet[Any]

  loadCache(set)

  override def addItem(item: Any): Unit = set.add(item)

  override def isUnique(item: Any): Boolean = !set.contains(item)
}
